{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from tqdm.notebook import tqdm\n",
    "from sklearn.preprocessing import MinMaxScaler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "file_dir = './pv_data/train_data_1.csv'\n",
    "file_dir_2 = './pv_data/train_data.csv'\n",
    "\n",
    "df = pd.read_csv(file_dir)\n",
    "#df = df[all_data_frame['ycsb']=='202106300288GF']\n",
    "print('数据总量',df.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 特征选择\n",
    "NUMERIC_COLS = ['gfrl',\n",
    "                'cur_a','cur_b','cur_c',\n",
    "                'vol_a','vol_b','vol_c',\n",
    "                'p', 'q', 'szgl', 'hgl', 'glys',\n",
    "                'low_tp', 'high_tp', 'avg_tp', \n",
    "                '地面气压(hPa)', '气温2m(℃)', '地表温度(℃)', '相对湿度(%)', '风速','总云量(tcc)', '净日照强度(net,J/m2)' \n",
    "                ]\n",
    "\n",
    "# 离散特征 ['sbbm', 'ycsb', 'hour']\n",
    "\n",
    "# 舍弃特征\n",
    "IGNORE_COLS = [\"index\", \"data_date\", \"time\", \"yhmc\", \"fdgl\", \"drfdl\", 'wind','weather']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 连续型特征 数值归一化\n",
    "selected_features = ['gfrl',\n",
    "                'cur_a','cur_b','cur_c',\n",
    "                'vol_a','vol_b','vol_c',\n",
    "                'p', 'q', 'szgl', 'hgl', 'glys',\n",
    "                'low_tp', 'high_tp', 'avg_tp', \n",
    "                '地面气压(hPa)', '气温2m(℃)', '地表温度(℃)', '相对湿度(%)', '风速','总云量(tcc)', '净日照强度(net,J/m2)']\n",
    "\n",
    "scaled_data = df[selected_features]\n",
    "scaler = MinMaxScaler()\n",
    "scaled_data = pd.DataFrame(scaler.fit_transform(scaled_data),columns=scaled_data.keys())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 合并数据\n",
    "df.drop(selected_features,axis=1,inplace=True)\n",
    "df = pd.concat([df,scaled_data], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfTest = df[df['time']==20211114]\n",
    "print('测试数据总量',dfTest.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfTrain = df[df['time']!=20211114]\n",
    "print('训练数据总量',dfTrain.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 打乱数据\n",
    "#dfTrain = dfTrain.sample(frac=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 切分数据 训练数据/测试数据\n",
    "# dfTrain = df\n",
    "# dfTest = df[30000:] "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#更具特征列 生成特征词典\n",
    "feature_dict = {}\n",
    "total_feature = 0\n",
    "for col in df.columns:\n",
    "    if col in IGNORE_COLS:\n",
    "        continue\n",
    "    elif col in NUMERIC_COLS:\n",
    "        feature_dict[col] = total_feature\n",
    "        total_feature += 1\n",
    "    else:\n",
    "        unique_val = df[col].unique()\n",
    "        feature_dict[col] = dict(zip(unique_val,range(total_feature,len(unique_val) + total_feature)))\n",
    "        total_feature += len(unique_val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfTrain = pd.concat([dfTrain, dfTrain])\n",
    "\n",
    "dfTrain = pd.concat([dfTrain, dfTrain])\n",
    "\n",
    "dfTrain = pd.concat([dfTrain, dfTrain])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "对训练集进行转化\n",
    "\"\"\"\n",
    "#生成标签列\n",
    "train_y = dfTrain[['fdgl']].values.tolist()\n",
    "\n",
    "dfTrain.drop(['fdgl','index'],axis=1,inplace=True)\n",
    "\n",
    "train_feature_index = dfTrain.copy()\n",
    "\n",
    "train_feature_value = dfTrain.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#生成特征索引矩阵与特征值矩阵\n",
    "for col in train_feature_index.columns:\n",
    "    if col in IGNORE_COLS:\n",
    "        train_feature_index.drop(col,axis=1,inplace=True)\n",
    "        train_feature_value.drop(col,axis=1,inplace=True)\n",
    "        continue\n",
    "    elif col in NUMERIC_COLS:\n",
    "        train_feature_index[col] = feature_dict[col]\n",
    "    else:\n",
    "        train_feature_index[col] = train_feature_index[col].map(feature_dict[col])\n",
    "        train_feature_value[col] = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "对测试集进行转化\n",
    "\"\"\"\n",
    "test_ids = dfTest['index'].values.tolist()\n",
    "dfTest.drop(['index'],axis=1,inplace=True)\n",
    "\n",
    "test_feature_index = dfTest.copy()\n",
    "test_feature_value = dfTest.copy()\n",
    "\n",
    "for col in test_feature_index.columns:\n",
    "    if col in IGNORE_COLS:\n",
    "        test_feature_index.drop(col,axis=1,inplace=True)\n",
    "        test_feature_value.drop(col,axis=1,inplace=True)\n",
    "        continue\n",
    "    elif col in NUMERIC_COLS:\n",
    "        test_feature_index[col] = feature_dict[col]\n",
    "    else:\n",
    "        test_feature_index[col] = test_feature_index[col].map(feature_dict[col])\n",
    "        test_feature_value[col] = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#test_feature_index.to_csv('./data/test_index.csv')\n",
    "\n",
    "#test_feature_value.to_csv('./data/test_value.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#dfTest.to_csv('./data/label.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "\"\"\"模型参数\"\"\"\n",
    "dfm_params = {\n",
    "    \"use_fm\":True,\n",
    "    \"use_deep\":True,\n",
    "    \"embedding_size\":8,\n",
    "    \"dropout_fm\":[0.1,0.1],\n",
    "    \"deep_layers\":[32,32],\n",
    "    \n",
    "    \"dropout_deep\":[0.1,0.1,0.1],\n",
    "    \"deep_layer_activation\":tf.nn.relu,\n",
    "    \"epoch\":50,\n",
    "    \"batch_size\":144,\n",
    "    \"learning_rate\":0.01,\n",
    "    \"optimizer\":\"adam\",\n",
    "    \"batch_norm\":0.5,\n",
    "    \"batch_norm_decay\":0.5,\n",
    "    \"l2_reg\":0,\n",
    "    \"verbose\":True,\n",
    "    \"eval_metric\":'gini_norm',\n",
    "    \"random_seed\":1\n",
    "}\n",
    "dfm_params['feature_size'] = total_feature\n",
    "dfm_params['field_size'] = len(train_feature_index.columns)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"开始建立模型\"\"\"\n",
    "#初始化占位变量 索引/值/标签\n",
    "feat_index = tf.placeholder(tf.int32,shape=[None,None],name='feat_index')\n",
    "feat_value = tf.placeholder(tf.float32,shape=[None,None],name='feat_value')\n",
    "label = tf.placeholder(tf.float32,shape=[None,1],name='label')\n",
    "\n",
    "\"\"\"建立weights\"\"\"\n",
    "weights = dict()\n",
    " \n",
    "# embeddings\n",
    "# 初始化embedding变量 大小为特征数量 * embedding大小\n",
    "weights['feature_embeddings'] = tf.Variable(\n",
    "                                tf.random_normal([dfm_params['feature_size'],\n",
    "                                dfm_params['embedding_size']],0.0,0.01),\n",
    "                                name='feature_embeddings')\n",
    "\n",
    "# embedding权重偏移量初始化 大小为特征数量 * 1\n",
    "weights['feature_bias'] = tf.Variable(tf.random_normal([dfm_params['feature_size'],1],0.0,1.0),name='feature_bias')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# deep layers\n",
    "num_layer = len(dfm_params['deep_layers'])\n",
    "num_layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# dnn网络权重偏置初始化\n",
    "# 输入大小 块大小*embedding大小 25*16\n",
    "\n",
    "input_size = 32\n",
    "\n",
    "glorot = np.sqrt(2.0/(input_size + dfm_params['deep_layers'][0]))\n",
    "\n",
    "weights['layer_0'] = tf.Variable(\n",
    "    np.random.normal(loc=0,scale=glorot,size=(input_size,dfm_params['deep_layers'][0])),dtype=np.float32\n",
    ")\n",
    "weights['bias_0'] = tf.Variable(\n",
    "    np.random.normal(loc=0,scale=glorot,size=(1,dfm_params['deep_layers'][0])),dtype=np.float32\n",
    ")\n",
    "\n",
    "for i in range(1,num_layer):\n",
    "    glorot = np.sqrt(2.0 / (dfm_params['deep_layers'][i - 1] + dfm_params['deep_layers'][i]))\n",
    "    weights[\"layer_%d\" % i] = tf.Variable(\n",
    "        np.random.normal(loc=0, scale=glorot, size=(dfm_params['deep_layers'][i - 1], dfm_params['deep_layers'][i])),\n",
    "        dtype=np.float32)  \n",
    "    weights[\"bias_%d\" % i] = tf.Variable(\n",
    "        np.random.normal(loc=0, scale=glorot, size=(1, dfm_params['deep_layers'][i])),\n",
    "        dtype=np.float32) \n",
    "\n",
    "glorot = np.sqrt(2.0/(input_size + 1))\n",
    "weights['concat_projection'] = tf.Variable(np.random.normal(loc=0,scale=glorot,size=(input_size,1)),dtype=np.float32)\n",
    "weights['concat_bias'] = tf.Variable(tf.constant(0.01),dtype=np.float32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"embedding\"\"\"\n",
    "embeddings = tf.nn.embedding_lookup(weights['feature_embeddings'],feat_index)\n",
    "\n",
    "reshaped_feat_value = tf.reshape(feat_value,shape=[-1,dfm_params['field_size'],1])\n",
    "\n",
    "embeddings = tf.multiply(embeddings,reshaped_feat_value)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "embeddings.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "embeddings = tf.reshape(embeddings,shape=[-1,1,200])\n",
    "embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def unit_lstm():\n",
    "    # 定义一层LSTM_CELL hiddensize 会自动匹配输入的X的维度\n",
    "    lstm_cell = tf.nn.rnn_cell.BasicLSTMCell(num_units=32, forget_bias=1.0, state_is_tuple=True)\n",
    "    \n",
    "    # 添加dropout layer， 一般只设置output_keep_prob\n",
    "    lstm_cell = tf.nn.rnn_cell.DropoutWrapper(cell=lstm_cell, input_keep_prob=1.0, output_keep_prob=1)\n",
    "    return lstm_cell\n",
    "\n",
    "# 调用MultiRNNCell来实现多层 LSTM\n",
    "mlstm_cell = tf.nn.rnn_cell.MultiRNNCell([unit_lstm() for i in range(1)], state_is_tuple=True)\n",
    "\n",
    "# 使用全零来初始化state\n",
    "init_state = mlstm_cell.zero_state(dfm_params['batch_size'], dtype=tf.float32)\n",
    "\n",
    "outputs, state = tf.nn.dynamic_rnn(mlstm_cell, inputs=embeddings, initial_state=init_state,time_major=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "outputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"deep part\"\"\"\n",
    "y_deep = tf.reshape(outputs,shape=[-1,32])\n",
    "y_deep"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(0,2):\n",
    "    y_deep = tf.add(tf.matmul(y_deep,weights[\"layer_%d\" %i]), weights[\"bias_%d\"%i])\n",
    "    y_deep = tf.nn.relu(y_deep)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_deep"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "out = tf.add(tf.matmul(y_deep,weights['concat_projection']),weights['concat_bias'],name=\"out\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_batch(Xi,Xv,y,batch_size,index):\n",
    "    start = index * batch_size\n",
    "    end = (index + 1) * batch_size\n",
    "    end = end if end < len(y) else len(y)\n",
    "    return Xi[start:end],Xv[start:end],y[start:end]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "\"\"\"loss and optimizer\"\"\"\n",
    "loss = tf.losses.mean_squared_error(label, out)\n",
    "#optimizer = tf.train.AdamOptimizer(learning_rate=dfm_params['learning_rate'], beta1=0.9, beta2=0.999,epsilon=1e-8).minimize(loss)\n",
    "\n",
    "optimizer = tf.train.AdamOptimizer(learning_rate=dfm_params['learning_rate']).minimize(loss)\n",
    "\n",
    "\"\"\"train\"\"\"\n",
    "\n",
    "sess  =tf.Session()\n",
    "sess.run(tf.global_variables_initializer())\n",
    "\n",
    "loss_polt = []\n",
    "for i in tqdm(range(dfm_params['epoch'])):\n",
    "    total_batch = int(len(train_y) / dfm_params['batch_size'])\n",
    "    for step in range(total_batch):\n",
    "        Xi_batch, Xv_batch, y_batch = get_batch(train_feature_index, train_feature_value, train_y, dfm_params['batch_size'], step)\n",
    "        epoch_loss,_ = sess.run([loss,optimizer],feed_dict={feat_index:Xi_batch,feat_value:Xv_batch,label:y_batch})\n",
    "        #print(\"Epoch %s,Step %s,Loss is %s\" % (str(i),str(step),str(epoch_loss)))\n",
    "    print(\"Epoch %s,Loss is %s\" % (str(i),str(epoch_loss)))\n",
    "    loss_polt.append(epoch_loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "epoch = [i for i in range(dfm_params['epoch'])]\n",
    "plt.figure(figsize=(10,10))\n",
    "plt.plot(epoch, loss_polt, label='epoch', color='blue',linewidth=1)#线1\n",
    "#plt.plot(id, res2, label='loss',color='green',linewidth=1)#线2\n",
    "plt.ylim(0,0.2)\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "test_feature_index = pd.read_csv('./data/test_index.csv', index_col=[0])\n",
    "test_feature_value = pd.read_csv('./data/test_value.csv', index_col=[0])\n",
    "label = pd.read_csv('./data/label.csv', index_col=[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "feed_dict={feat_index:test_feature_index,feat_value:test_feature_value}\n",
    "\n",
    "pre = sess.run(out,feed_dict)\n",
    "\n",
    "res = []\n",
    "for i in pre:\n",
    "    res.append(float(i) if i>0 else abs(i))\n",
    "    #res.append(i)\n",
    "\n",
    "res2 = label['fdgl'].tolist()\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "id = [i for i in range(144)]\n",
    "\n",
    "plt.figure(figsize=(40,5))\n",
    "plt.plot(id, res, label='l1', color='blue',linewidth=1)#线1\n",
    "plt.plot(id, res2, label='l2',color='green',linewidth=1)#线2\n",
    "plt.ylim(0,5)\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "saver = tf.train.Saver()\n",
    "saver.save(sess, './model/my_test_model')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sess.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:tf1.14]",
   "language": "python",
   "name": "conda-env-tf1.14-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
